# 实现包的增量更新
import datetime
import hashlib
import threading
import time
import concurrent.futures
from urllib import request
import urllib.error
from globally import server_version, LocalDataRootPath
import os, requests, sys, copy
from lxml import html
import urllib.parse
from Class_Log import Log
from concurrent.futures import ThreadPoolExecutor
from Class_PackSHA256 import PackSHA256

class _LocalPackList:
    # 同过源地址repo_url获取对应本地存放路径
    @classmethod
    def _local_datapath(cls, repo_url):
        if not repo_url.endswith('/'):
            repo_url = repo_url + '/'
        local_datapath = LocalDataRootPath + '/'.join((repo_url+'Packages').split('/')[3:])
        if not os.path.exists(local_datapath):
            os.makedirs(local_datapath)
        return local_datapath

    # 通过repo_url获取对应本地路径所有包列表
    def local_packlist(self, repo_url):
        local_exist_packlist = []
        for root, dirname, filenames in os.walk(self._local_datapath(repo_url)):
            for mumber in filenames:
                if mumber.endswith(".rpm"):
                    local_exist_packlist.append(mumber)
        return local_exist_packlist


class _OnlinePackList:
    def online_packlist(self, repo_url):
        if not repo_url.endswith('/'):
            repo_url = repo_url + '/'
        url = repo_url + 'Packages'
        response = requests.get(url)
        if response.status_code == 200:
            html_content = response.text
            # soup = BeautifulSoup(html_content, 'html.parser')
            # html = soup.prettify()
        else:
            print("外网源访问错误")
            sys.exit(1)
        decoded_html = urllib.parse.unquote(html_content, encoding="UTF-8")
        online_exist_packlist = []
        # sys.exit(0)
        tree = html.fromstring(html_content)
        links = tree.xpath('//a/@href')
        for mumber in links:
            if mumber.endswith(".rpm"):
                online_exist_packlist.append(mumber.replace("%2B", "+"))
        # print(packlist)
        # print("外网源包总计：", len(packlist))
        return online_exist_packlist

# 任务书平台，用于创建任务书
class AssignmentPlat(_LocalPackList, _OnlinePackList, Log):
    def create_assignments(self, version) -> list:
        """
        name: create_assignments # 创建任务书（对外接口)
        params：version系统版本
        return： 任务书列表（list）
        detail：version版本通过_assignment_list得到任务书。并写入日志
            任务书格式：
                {"repo_url": "", # 外网地址 (举例：https://update.cs2c.com.cn/NS/V10/V10SP2/os/adv/lic/base/x86_64/）
                 "local_workpath": "", # 对应本地存储地址 （举例： .../database/update.cs2c.com.cn/NS/V10/V10SP2/os/adv/lic/base/x86_64/Packages/）
                 "task_removepacklist": "", # 需要从本地移除的软件包
                 "task_downloadpacklist": ""    # 需要从外网下载的软件包
                }
        """
        self.LOGAssignment(1, "\n****************************************\n  增量更新  版本：{} 时间：{}\n****************************************".format(version, datetime.datetime.now()))
    # ---- 获取对应version的所有任务书 ----
        assignment_list = self._assignment_list(version)
    # ---- 日志记录 ----
        for assignment in assignment_list:
            self.LOGAssignment(1, '====> {}的任务书： {} <===='.format(version, assignment["repo_url"]))
            loginfo = ["    外网源地址：{}".format(assignment["repo_url"]),
                       "    内网地址：{}".format(assignment["local_workpath"]),
                       "    {}".format("需要从外网源下载：{}个包，需要从内网删除：{}个包".format(len(assignment["task_downloadpacklist"]), len(assignment["task_removepacklist"])))]
            self.LOGAssignment(1, loginfo)
            for mumber in assignment["task_downloadpacklist"]:
                self.LOGAssignment(1, "        新增： {}".format(mumber))
            for mumber in assignment["task_removepacklist"]:
                self.LOGAssignment(1, "        删除： {}".format(mumber))
        return assignment_list

    # 创建任务书，用来记录，软件包的增加于删除。 输入：系统版本 输出：任务书（list） —— 记录url、本地工作目录、任务细则
    def _assignment_list(self, version) -> list:
        # 生成每个源地址的任务书。
        # global _assignment
        assignment_list = []
        for repo_url in server_version[version]:
        # ---- 前置准备工作 ----
            # 1、规范repo_url格式，以/结尾
            if not repo_url.endswith('/'):
                repo_url = repo_url + '/'
            # 2、创建对应repo_url的本地目录
            # local_workpath为repo_url对应本地的工作目录，和Packages，repodata的目录
            _local_workpath = LocalDataRootPath + '/'.join((repo_url+"Packages/").split('/')[3:])
            if not os.path.exists(_local_workpath):
                os.makedirs(_local_workpath)
        # ---- 任务书创建 ----
            _assignment = {"repo_url": repo_url, # https://update.cs2c.com.cn/NS/V10/V10SP2/os/adv/lic/base/x86_64/
                           "local_workpath": _local_workpath, # .../database/update.cs2c.com.cn/NS/V10/V10SP2/os/adv/lic/base/x86_64/Packages/
                           "task_removepacklist": "",
                           "task_downloadpacklist": ""}
            _assignment["task_removepacklist"], _assignment["task_downloadpacklist"] = self._pack_compare(_assignment["repo_url"])
            assignment_list.append(_assignment)
        return assignment_list

    # 内外网源包对比。
    @classmethod
    def _pack_compare(cls, repo_url: str) -> list:
        localpacks = _LocalPackList().local_packlist(repo_url)
        onlinepacks = _OnlinePackList().online_packlist(repo_url)
        localpackleave = copy.deepcopy(localpacks)
        onlinepackleave = copy.deepcopy(onlinepacks)
        for mumber in onlinepacks:
            if mumber in localpacks:
                localpackleave.remove(mumber)
                onlinepackleave.remove(mumber)
        # 生成结论
        # result_info = "需要从外网源下载：{}个包，需要从内网源删除：{}个包".format(len(onlinepackleave), len(localpackleave))
        result_removelist = localpackleave
        result_downloadlist = onlinepackleave
        # for packname in localpackleave:
        #     re_pack = "- {}".format(packname)
        #     compare_result.append(re_pack)
        # for packname in onlinepackleave:
        #     re_pack = "+ {}".format(packname)
        #     compare_result.append(re_pack)
        return [result_removelist, result_downloadlist]

class Sha256CheckError(Exception):
    def __init__(self, message):
        self.message = message
        super().__init__(self.message)

# 执行任务书
class ExecPlat(Log):
    @classmethod
    def _remove(cls, assignment):
        if assignment["task_removepacklist"]:
            for _pack in assignment["task_removepacklist"]:
                _path = assignment["local_workpath"] + _pack
                os.remove(_path)
                print("删除软件包： ", _pack)
        cls.LOGExec("====> 模块: 执行任务书（删除） <====  任务地址：{}，删除数量：{} <====".format(assignment["repo_url"], len(assignment["task_removepacklist"])))

    @classmethod
    def _download(cls, assignment):
        def task(count, totalcount, packname, check_sha256=None):
            """
            :param packname:
            :param check_sha256:
            :return:  {packname: status} # status: 0——状态正常。1——状态异常，下载过程中存在异常。2——状态异常，官方repodata中没有该包的校验值
            """
            # 整理下载地址和存储路径
            if "+" in packname:
                real_packname = packname.replace("+", "%2B")
            else:
                real_packname = packname
            downloadurl = assignment["repo_url"] + "Packages/" + real_packname
            destpath = assignment["local_workpath"] + packname
            with print_lock:
                print("下载任务{}/{} —— 包名：{}, 正确校验值：{}".format(count, totalcount, packname, check_sha256), flush=True)
            # 进行下载及sha256验证
            while True:
                error_times = 0
                try:
                    if error_times <= 3: # 错误次数小于三次
                        opener = urllib.request.build_opener()
                        opener.addheaders = [('User-Agent', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1941.0 Safari/537.36')]
                        urllib.request.install_opener(opener)
                        urllib.request.urlretrieve(downloadurl, destpath)
                        time.sleep(0.5)
                        with open(destpath, "rb") as f:
                            sha256_hash = hashlib.sha256()
                            for chunk in iter(lambda: f.read(4096), b""):
                                sha256_hash.update(chunk)
                        if not check_sha256:    # 软件包不存在官方sha256
                            return {packname: 2}
                        if sha256_hash.hexdigest() == check_sha256:
                            return {packname: 0}
                        else:
                            raise Sha256CheckError
                    else:   # 错误次数超过3次
                        return {packname: 1}
                except:
                    error_times = error_times + 1
                    if os.path.exists(destpath):
                        os.remove(destpath)

        totalcount = len(assignment["task_downloadpacklist"])   # 需要下载包的总数量
        # ---- 获取url对应的所有sha256数值 ----
        print("正在获取软件包校验信息： url={}".format(assignment["repo_url"]))
        sha256_list = PackSHA256().task(assignment["repo_url"])
        # ---- 创建线程池，多线程下载包 ----
        count = 0
        futures = []
        pool = ThreadPoolExecutor(30)
        print_lock = threading.Lock()
        for packname in assignment["task_downloadpacklist"]:
            if packname not in sha256_list.keys():
                result_sha256 = None
            else:
                result_sha256 = sha256_list[packname]   # 软件包的sha256的正确校验值。
            count = count + 1
            future = pool.submit(task, count, totalcount, packname,result_sha256)
            futures.append(future)

        concurrent.futures.wait(futures)
        download_results = {} # 所有下载的结果
        for future in futures:
            download_results.update(future.result())
        pool.shutdown()

        # ---- 结果整理 ----
        download_success_list = []
        download_error_list = []
        download_nosha256_list = []
        for mumber in download_results.keys():
            if download_results[mumber] == 0:
                download_success_list.append(mumber)
            if download_results[mumber] == 1:
                download_error_list.append(mumber)
            if download_results[mumber] == 2:
                download_nosha256_list.append(mumber)
        print("下载完成：总共需要下载 {} 个：其中{}个下载成功， {}个下载失败， {}个无官方SHA256".format(len(assignment["task_downloadpacklist"]), len(download_success_list), len(download_error_list), len(download_nosha256_list)))
        # ---- 日志记录 ----
        cls.LOGExec("====> 模块: 执行任务书（下载） <==== 任务地址：{} 总共需要下载{}个  其中{}个下载成功， {}个下载失败， {}个无官方SHA256".format(assignment["repo_url"], len(assignment["task_downloadpacklist"]), len(download_success_list), len(download_error_list), len(download_nosha256_list)))
        cls.LOGExec("下载失败列表： {}".format(download_error_list))
        cls.LOGExec("下载无官方SHA256列表： {}".format(download_nosha256_list))

    @classmethod
    def handle_assignments(cls, assignment_lists):
        """
        name: handle_assignments # 处理任务书
        params: 任务书列表（list）
        detail：
        """

        for assignment in assignment_lists:   # 判断本地存放包的地址是否存在，不存在则创建。
            # _localpackdir = assignment["local_workpath"] + "Packages/"
            # if not os.path.exists(_localpackdir):
            #     os.makedirs(_localpackdir)
        # 分别执行删除任务和下载任务
            cls._remove(assignment)
            cls._download(assignment)




